import facade
from xjlibrary.our_file_dir import BaseDir

curPath = BaseDir.get_file_dir_absolute(__file__)
TopPath = BaseDir.get_upper_dir(curPath, -2)
sPath = BaseDir.get_new_path(TopPath, "download", "rucjournal", "download", "listpage")
BaseDir.create_dir(sPath)


class DownListPage(object):
    def __init__(self):
        self.proxy = {
            "http": "192.168.30.176:8012",
            "https": "192.168.30.176:8012"
        }

    def downpage(self, pagesize):
        """
        下载某一页
        :param pagesize:
        :return:
        """
        headers = {
            'referer': 'https://www.rdfybk.com/qw',
            "origin": "https://www.rdfybk.com",
            'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
            'X-Requested-With': 'XMLHttpRequest',
            'Accept': '*/*',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Accept-Encoding': 'gzip, deflate',
            'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)',
        }
        postData = {
            'data': '{"TreeCode":"09,01,08,02,06,03,04,07,05","StartYear":"1995","EndYear":"2019","ListSearchCondition":[],"Order":"2","PageSize":20,"PageIndex":%s,"IsBuy":0}' % str(
                pagesize)}
        url = 'https://www.rdfybk.com/Qw/Search'
        BoolResult, errString, r = facade.BaseRequestPost(url,
                                                          data=postData,
                                                          proxies=self.proxy,
                                                          headers=headers,
                                                          endstring="",
                                                          verify=False,
                                                          timeout=60)
        if BoolResult:
            return r.text
        else:
            return False

    def run(self):
        i = 1
        # 页码会变
        while i <= 28811:
            htmlText = self.downpage(i)
            if htmlText:
                htmlText = htmlText.replace("\n", "").replace("\r", "").replace("\0", "") + "\n"
                BaseDir.single_add_file(BaseDir.get_new_path(sPath, "result.big_json"), htmlText)
            else:
                continue
            i = i + 1
            print("当前下载到了{}页".format(i))


if __name__ == "__main__":
    down = DownListPage()
    down.run()
